# -*- coding: utf-8 -*-
'''
Created on 27.02.2015

@author: MathiasAr
'''

import os, sys, traceback, codecs
from pyquery import PyQuery as pq


def getPkgInfos(fpath):
    """
    pkgName, pkgTitle, pkgDesc, pyQuery = getPkgInfos(<../pkg-html-file>)
    """
    fpathsplit = fpath.split('#')
    if (len(fpathsplit) > 1):
        fpath = '#'.join(fpathsplit[:len(fpathsplit) - 1])
        
    print "extract package info from: %s" % fpath
    d = pq(codecs.open(fpath, 'r', 'iso-8859-1').read())
    sect = d.find('div.package').parent('div')
    if not sect:
        print "file contains no top level package instructions, ignore it"
        return None, None, None, None
    
    pkgName = sect.children('h1').find('a').eq(0)
    if not pkgName or not pkgName.attr('name'):
        print "No package name found in headline... abort"
        return None, None, None, sect
    
    pkgNameVers = pkgName.parent('h1')
    if not pkgNameVers:
        print "No package title found in headline... abort"
        return None, None, None, sect
    
    pkgName = pkgName.attr('name').strip()
    pkgNameVers = unicode(pkgNameVers.text()).split(' ')[0]
    if not "-" in pkgNameVers:
        bookVers = unicode(d('body').attr('id')).split('-')
        if len(bookVers) > 1:
            bookVers = '.'.join(bookVers[1:])
        else:
            bookVers = bookVers[0]
        pkgNameVers = "%s-%s" % (pkgName, bookVers)
    
    pkgDesc = sect.children('div.package').eq(0).children('p')
    if not pkgDesc:
        print "No package description found under <div.package>"
        return None, None, None, sect
    
    return pkgName, pkgNameVers, unicode(pkgDesc.text()).strip(), sect


def scanHtml(fpath, pkgName, pkg, outdir):
    fdir = os.path.split(fpath)[0]
    print "%s --> scanHtml: %s" % (pkgName, fpath)
        
    try:
        if not os.path.exists(outdir):
            os.makedirs(outdir)
        
        # get package info blocks with download links
        urlBlocks = pkg.children('div.package').children('div.itemizedlist')
        if not urlBlocks or len(urlBlocks) == 0:
            print "No URL information found for package %s in %s" % (pkgName, fpath)
        else:    
            wget = codecs.open(os.path.join(outdir, 'wget-list'), 'w', 'utf-8')
            for b in range(len(urlBlocks)):
                block = urlBlocks.eq(b)
                # title of the block
                wget.write(u'#########################\n# %s\n' % unicode(block.prevAll('h3').text()).replace('\n', '\n#'))
                url = block.find('a.ulink')
                for u in url:
                    u = url(u)
                    href = unicode(u.attr('href')).split(' ')[-1]
                    wget.write('# %s\n%s\n' % (unicode(u.parent('p').text()).split(': ')[0].strip().replace('\n', '\n#'), href))
    
            wget.close()
        
        # get dependencies
        reqDeps = pkg.children('div.package').children('p.required').find('a.xref')
        recDeps = pkg.children('div.package').children('p.recommended').find('a.xref')
        optDeps = pkg.children('div.package').children('p.optional').find('a.xref')
        if not reqDeps and not recDeps and not optDeps:
            print "No dependency information found for package %s in %s" % (pkgName, fpath)
        else:
            depspkg = codecs.open(os.path.join(outdir, 'deps.pkg'), 'w', 'utf-8')
            if reqDeps:
                depspkg.write('#########################\n# required dependencies\n')
                for r in reqDeps:
                    r = reqDeps(r)
                    xref = r.attr('href').replace('/', os.path.sep)
                    depname, depnversion, depdesc, depelem = getPkgInfos(os.path.join(fdir, xref))
                    if depname and depnversion:
                        depspkg.write('# package %s (%s)\n%s   %s\n' % (depnversion, xref, depname, depnversion.split('-')[-1]))
                    else:
                        print "Can not resolve xref: %s" % os.path.join(fdir, xref)
                        depspkg.write('# package %s\n#%s\n' % (unicode(r.text()).replace('\n', ' '), r.attr('href')))
                    
            if recDeps: 
                depspkg.write('#########################\n# recommended dependencies\n')
                for r in recDeps:
                    r = recDeps(r)
                    xref = r.attr('href').replace('/', os.path.sep)
                    depname, depnversion, depdesc, depelem = getPkgInfos(os.path.join(fdir, xref))
                    if depname and depnversion:
                        depspkg.write('# package %s (%s)\n%s   %s\n' % (depnversion, xref, depname, depnversion.split('-')[-1]))
                    else:
                        print "Can not resolve xref: %s" % os.path.join(fdir, xref)
                        depspkg.write('# package %s\n#%s\n' % (unicode(r.text()).replace('\n', ' '), r.attr('href')))
            
            if optDeps:
                depspkg.write('#########################\n# optional dependencies\n')
                for r in optDeps:
                    r = optDeps(r)
                    xref = r.attr('href').replace('/', os.path.sep)
                    depname, depnversion, depdesc, depelem = getPkgInfos(os.path.join(fdir, xref))
                    if depname and depnversion:
                        depspkg.write('# package %s (%s)\n#%s   %s\n' % (depnversion, xref, depname, depnversion.split('-')[-1]))
                    else:
                        print "Can not resolve xref: %s" % os.path.join(fdir, xref)
                        depspkg.write('# package %s\n#%s\n' % (unicode(r.text()).replace('\n', ' '), r.attr('href')))
        
            depspkg.close()
            
        # get configuration, make and install information
        install = codecs.open(os.path.join(outdir, 'install.txt'), 'w', 'utf-8')
        div = pkg.children('div.package')
        while div:
            div = div.next()
            if not div.filter('div'):
                continue
            
            if div.hasClass('kernel'):
                # get kernel parameter
                kernel = div
                kernelConfigs = kernel('code')
                if kernelConfigs:
                    install.write('#########################\n# kernel configuration hints\n')
                    for kconf in kernelConfigs:
                        kconf = kernelConfigs(kconf)
                        install.write('# %s\n' % unicode(kconf.parent('pre').prev('p').text()).replace('\n', '\n#'))
                        install.write('# %s\n' % unicode(kconf.text()).replace('\n', '\n#'))
                install.write('\n\n')
                
            elif div.hasClass('installation'):
                # get install instructions:
                inst = div
                instCmds = inst('kbd.command')
                if instCmds:
                    install.write('#########################\n# make and install instructions\n')
                    for cmd in instCmds:
                        cmd = instCmds(cmd)
                        install.write('# %s\n' % unicode(cmd.parent('pre').prev('p').text()).replace('\n', '\n#'))
                        install.write('%s\n\n' % cmd.text())
                
            elif div.hasClass('commands'):
                # get install command explanations:
                commands = div
                cmdExpl = commands('p')
                if cmdExpl:
                    n = 1
                    install.write('\n\n# install description:\n')
                    for expl in cmdExpl:
                        expl = cmdExpl(expl)
                        install.write('#  (%s) %s\n' % (unicode(n), unicode(expl.text()).replace('\n', '\n#')))
                        n += 1
                
                install.write('\n\n')
                
            elif div.hasClass('configuration'):
                # get configuration hints
                configs = div.children('div')
                if not configs:
                    configs = div
                
                n = 1
                install.write('#########################\n# configuration instructions\n')
                for config in configs:
                    config = configs(config)
                    install.write('# --> %s\n' % unicode(config.children().eq(0).text()).replace('\n', '\n#'))
                    for hint in range(1, len(config.children())):
                        hint = config.children().eq(hint)
                        cmds = hint.find('kbd.command')
                        if cmds:
                            for c in cmds:
                                c = cmds(c)
                                install.write('%s\n' % c.text())
                            install.write('\n')
                        else:
                            install.write('#  (%s) %s\n' % (str(n), unicode(hint.text()).replace('\n', '\n#')))
                            n += 1
                install.write('\n\n')
            else:
                # get other hints
                etcs = div.children('div')
                if not etcs:
                    etcs = div
                
                n = 1
                install.write('#########################\n# other hints (%s):\n' % unicode(div.children().eq(0).text()))
                for etc in etcs:
                    etc = etcs(etc)
                    install.write('# --> %s\n' % unicode(etc.children().eq(0).text()).replace('\n', '\n#'))
                    for hint in range(1, len(etc.children())):
                        hint = etc.children().eq(hint)
                        cmds = hint.find('kbd.command')
                        if cmds:
                            for c in cmds:
                                c = cmds(c)
                                install.write('%s\n' % c.text())
                            install.write('\n')
                        else:
                            install.write('#  (%s) %s\n' % (str(n), unicode(hint.text()).replace('\n', '\n#')))
                            n += 1
                install.write('\n\n')
            # end while
            pass
        
        install.close()
        
    except Exception as e:
        print e
        type, value, tb = sys.exc_info()
        traceback.print_tb(tb)
        return False
    
    return True
    
    pass








if __name__ == '__main__':
    
    target = os.path.join("..", "..", "..", "from_html")
    # root = r"C:\Users\MathiasAr\Documents\www\BLFS\www.linuxfromscratch.org\blfs\view\stable"
    root = os.path.join("C:\\", "Users", "MathiasAr", "Documents", "www", "BLFS_Current", "www.linuxfromscratch.org", "blfs", "view", "svn")
    
    failedFiles = []
    failedScanPkgs = []
    pkgList = codecs.open(os.path.join(target, "package.list"), "w", 'utf-8')
    pkgList.write('# packages found in HTML sources under %s\n' % root)
    
    for d in os.listdir(root):
        dpath = os.path.join(root, d)
        if os.path.isdir(dpath):
            print "Scan: %s" % dpath
            for f in os.listdir(dpath):
                fpath = os.path.join(dpath, f)
                if os.path.isfile(fpath) and '.html' in f:
                    pkgName, pkgTitle, pkgDesc, pkgHtmlElm = getPkgInfos(fpath)
                    if not pkgName:
                        failedFiles.append(fpath)
                    elif not scanHtml(fpath, pkgName, pkgHtmlElm, os.path.join(target, pkgName)):
                        failedScanPkgs.append(pkgName)
                    else:
                        # OK, append file info
                        pkgVersion = pkgTitle.split('-')[-1]
                        pkgTitle = "%s-%s" % (pkgName, pkgVersion)
                        pkgArchive = None
                        if os.path.isfile(os.path.join(target, pkgName, 'wget-list')):
                            wget = file(os.path.join(target, pkgName, 'wget-list'), 'r')
                            for archive in wget.readlines():
                                if archive != "" and not archive.startswith('#'):
                                    archive = archive.split('/')[-1].strip()
                                    if (archive.endswith('.tar') or
                                        archive.endswith('.tgz') or
                                        archive.endswith('.tar.gz') or
                                        archive.endswith('.tar.xz') or
                                        archive.endswith('.tar.bz2')):
                                        pkgArchive = archive[:archive.rfind('.t')]
                                        break
                            wget.close()
                        if not pkgArchive:
                            print "%s - pkg info: can not find a valid package archive file..." % pkgName
                            pkgArchive = pkgTitle
                        oneLineDesc = ""
                        for desc in unicode(pkgDesc).splitlines():
                            oneLineDesc += desc.replace('\t', ' ').strip() + " "
                        infoFile = codecs.open("%s.info" % os.path.join(target, pkgName, pkgName), "w", 'utf-8')
                        infoFile.write(u'%s %s %s' % (unicode(pkgTitle).ljust(20), unicode(pkgArchive).ljust(31), oneLineDesc))
                        infoFile.close()
                        pkgList.write(u'%s %s %s %s\n' % (unicode(pkgName).ljust(30), unicode(pkgArchive).ljust(40), unicode(pkgVersion).ljust(20), oneLineDesc))
                pass

    pkgList.close()
    print "--------------------------------"
    for f in failedFiles:
        print f
    print "--------------------------------"
    for p in failedScanPkgs:
        print p
    pass


    
    

